import os
from typing import List
import threading
import tiktoken
from tqdm import trange
import time
import requests
import random
import json
# from langchain.document_loaders import PyPDFLoader

class tokenCounter():

    def __init__(self) -> None:
        self.encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")
        self.model_price = {}
        
    def num_tokens_from_string(self, string:str) -> int:
        return len(self.encoding.encode(string))

    def num_tokens_from_list_string(self, list_of_string:List[str]) -> int:
        num = 0
        for s in list_of_string:
            num += len(self.encoding.encode(s))
        return num
    
    def compute_price(self, input_tokens, output_tokens, model):
        return (input_tokens/1000) * self.model_price[model][0] + (output_tokens/1000) * self.model_price[model][1]

    def text_truncation(self,text, max_len = 1000):
        encoded_id = self.encoding.encode(text, disallowed_special=())
        return self.encoding.decode(encoded_id[:min(max_len,len(encoded_id))])

# def load_pdf(file, max_len = 1000):
#     loader = PyPDFLoader(file)
#     pages = loader.load_and_split()
#     encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")
#     text = ''.join([p.page_content for p in pages])
#     return encoding.decode(encoding.encode(text)[:max_len])
import re
def sanitize_folder_name(topic):
    """
    替换不适合作为文件夹名称的字符。
    
    参数:
    - topic: 原始的topic字符串
    
    返回:
    - 处理后的适合作为文件夹名称的字符串
    """
    # 定义非法字符的正则表达式
    illegal_chars = r'[<>:"/\\|?*\x00-\x1F]'  # 包括Windows和Linux/macOS的非法字符
    # 使用正则表达式替换非法字符为下划线
    sanitized_topic = re.sub(illegal_chars, '_', topic)
    # 移除连续的下划线
    sanitized_topic = re.sub(r'__+', '_', sanitized_topic)
    # 移除开头和结尾的下划线
    sanitized_topic = sanitized_topic.strip('_')
    
    # 如果结果为空，则返回默认名称
    if not sanitized_topic:
        sanitized_topic = "default_folder"
    
    return sanitized_topic